#include "DPTest.h"
#include "../Scene/ZDataManager.h"
#include "../Tools/FileTools.h"
#include "../Tools/XmlParser.h"

GLOBAL_NAMESPACE_BEGIN

namespace MachineLearningTest
{
    const char* DPTest::InputOptionNames[] = {
        "type"
        "InputTxtFile",
        "OutputFolder",
        "OutputName",
        "dcScale",
        "gapScale",
        "reCalculatedDG",
        "skip"
    };

    void DPTest::runTest()
    {
        //Log::Stream stream = LogManager::getSingletonPtr()->getDefaultLog()->stream();
        DebugLog << "Begin to run density-peak algorithm..\n";
        MLAlgorithms::DensityPeak dpAlgo;
        dpAlgo.readFromFile("../data/randomPoints.txt");
        dpAlgo.setSCScaleParam();
        dpAlgo.commit();
        dpAlgo.saveResultToFile("../data/results.txt");
        DebugLog << "Finish.\n";

        saveAsObj("../data/points.obj", &dpAlgo);
        saveResultAsObj("../data/results.obj", &dpAlgo);

        SaveOptions opt;
        opt.bDelta = false;
        opt.bDensity = true;
        opt.threshold = 0.0;
        analyseAndSaveResultsAsObj("../data/pointsAnaDensity.obj", "../data/resultAnaDensity.obj", &dpAlgo, opt);
        opt.bDelta = true;
        opt.bDensity = false;
        analyseAndSaveResultsAsObj("../data/pointsAnaDelta.obj", "../data/resultAnaDelta.obj", &dpAlgo, opt);
        opt.bDensity = true;
        analyseAndSaveResultsAsObj("../data/pointsAna.obj", "../data/resultAna.obj", &dpAlgo, opt);
        opt.threshold = 0.6;
        analyseAndSaveResultsAsObj("../data/pointsAnaT.obj", "../data/resultAnaT.obj", &dpAlgo, opt);
    }

    void DPTest::runTest2()
    {
        DebugLog << "***********************";
        DebugLog << " Run Test 2";
        std::string inFileName("E:\\works\\learning\\data\\cmu_pie\\cmu_pie.txt");
        std::string folderName("E:\\works\\learning\\data\\cmu_pie\\");
        std::vector<std::string> fileNames;
        //std::string fileName("E:\\\work\\LearningAlgorithms\\data\\cmu_pie\\cmu_pie\\1.jpg");
        //ImageSpace::Image *ima = new ImageSpace::Image;
        //ima->loadImageFromFile(fileName.c_str());
        //DataManager::getSingletonPtr()->addImage(ima);
        getDataFileNames(inFileName.c_str(), folderName, fileNames);
        for (int i = 0; i < fileNames.size(); i++)
        {
            ImageSpace::Image *ima = new ImageSpace::Image;
            ima->loadImageFromFile(fileNames.at(i).c_str());
            DataManager::getSingletonPtr()->addImage(ima);
        }

        // change image to vector
        size_t nSize = DataManager::getSingletonPtr()->getImageCount();
        size_t nDim = DataManager::getSingletonPtr()->getImageById()->getDimension();
        Eigen::MatrixXd data(nSize, nDim);
        double* imaData = new double[nDim];
        for (int i = 0; i < nSize; i++)
        {
            ImageSpace::Image* ima = DataManager::getSingletonPtr()->getImageById(i);
            if (!ima) continue;
            ima->getData(imaData);
            for (int j = 0; j < nDim; j++)
                data(i,j) = imaData[j];
            //DebugLog << data;
        }

        delete [] imaData;
        DebugLog << "Data has loaded.";
        //DefaultLog << data;
        MLAlgorithms::DensityPeak dpAlgo;
        DebugLog << "Begin to run density-peak algorithm..\n";
        dpAlgo.init(data);
        dpAlgo.setSCScaleParam(30);
        dpAlgo.commit();
        std::string outFileName = folderName + "/result.txt";
        dpAlgo.saveResultToFile(outFileName.c_str());
        std::string dgFileName = folderName + "/decisionGraph.obj";
        SaveOptions opt = { false, true, true, 0.0 };
        saveDecisionGraphAsObj(dgFileName.c_str(), opt, &dpAlgo);
        DebugLog << "Finish.\n";
    }

    void DPTest::runTest3()
    {
        DebugLog << "***********************";
        DebugLog << " Run Test 3";

        std::string fileRetName("E:\\works\\learning\\data\\cmu_pie\\result.txt");

        MLAlgorithms::DensityPeak dpAlgo;
        dpAlgo.readResultFromFile(fileRetName.c_str());
        dpAlgo.analyseResult();
    }

    void DPTest::runDPAlgorithm(const char* dataTxtFile)
    {
        DebugLog << "******************************";
        DebugLog << " Run Density Peak Algorithm..";
        std::string inFileName(dataTxtFile);
        std::string folderName(inFileName.substr(0, inFileName.find_last_of("/\\")));
        std::vector<std::string> fileNames;
        getDataFileNames(inFileName.c_str(), folderName, fileNames);
        DebugLog << " 1. Load data.";
        for (int i = 0; i < fileNames.size(); i++)
        {
            ImageSpace::Image *ima = new ImageSpace::Image;
            ima->loadImageFromFile(fileNames.at(i).c_str());
            DataManager::getSingletonPtr()->addImage(ima);
        }

        // change image to vector
        int nSize = DataManager::getSingletonPtr()->getImageCount();
        int nDim = DataManager::getSingletonPtr()->getImageById()->getDimension();
        Eigen::MatrixXd data(nSize, nDim);
        double* imaData = new double[nDim];
        for (int i = 0; i < nSize; i++)
        {
            ImageSpace::Image* ima = DataManager::getSingletonPtr()->getImageById(i);
            ima->getData(imaData);
            for (int j = 0; j < nDim; j++)
                data(i, j) = imaData[j];
            //DebugLog << data;
        }
        delete[] imaData;
        DebugLog << " 2. Data has loaded.";

        //DefaultLog << data;
        MLAlgorithms::DensityPeak dpAlgo;
        DebugLog << " 3. Begin to run density-peak algorithm..\n";
        dpAlgo.init(data);
        dpAlgo.setSCScaleParam(30);
        dpAlgo.commit();
        std::string outFileName = folderName + "/result.txt";
        dpAlgo.saveResultToFile(outFileName.c_str());
        std::string dgFileName = folderName + "/decisionGraph.obj";
        SaveOptions opt = { false, true, true, 0.0 };
        saveDecisionGraphAsObj(dgFileName.c_str(), opt, &dpAlgo);
        DebugLog << "Finish";
    }

    void DPTest::analyseDPResult(const char* dataTxtFile, const char* resultFile, const char* outputFolder)
    {
        DebugLog << "***************************";
        DebugLog << " Analyse the Result data";

        std::string fileRetName(resultFile);
        MLAlgorithms::DensityPeak dpAlgo;
        dpAlgo.setGapParam(50);
        dpAlgo.readResultFromFile(fileRetName.c_str());
        dpAlgo.analyseResult();

        std::string dataFileFolder(fileRetName.substr(0, fileRetName.find_last_of("/\\")));
        std::string inFileName(dataTxtFile);
        std::vector<std::string> fileNames;
        getDataFileNames(inFileName.c_str(), dataFileFolder, fileNames);

        DebugLog << " output the clustered data..";
        std::vector<int> clusterIdx = dpAlgo.getClusterIdx();
        for (size_t i = 0; i < clusterIdx.size(); i++)
        {
            int id = clusterIdx[i];
            std::string srcFile(fileNames[id]);
            std::stringstream ss;
            ss << outputFolder << "/" << i << "_" << (srcFile.substr(srcFile.find_last_of("/\\")+1));
            FileTools::copyFile(srcFile.c_str(), ss.str().c_str());
        }
    }

    void DPTest::runTest4()
    {
        DebugLog << "***********************";
        DebugLog << " Run Test 4";

        const char* fileNameVecs[] = { "E:\\work\\LearningAlgorithms\\data\\fw_32_32",
                                       "E:\\work\\LearningAlgorithms\\data\\cmu_pie\\cmu_pie\\cmu_pie.txt"};

        InputOptions inOpt;
        inOpt.txtFileFullPath = std::string("E:\\work\\LearningAlgorithms\\data\\fw_32_32\\fw_32_32\\fw_32_32_1.txt");
        inOpt.outputFolder = std::string("E:\\work\\LearningAlgorithms\\data\\fw_32_32\\output");
        inOpt.resultNamePrefix = std::string("\\ret5");
        inOpt.dcScale = 0.04;
        inOpt.gapScale = 1;
        inOpt.reCalculateDG = true;

        std::string inFileName(inOpt.txtFileFullPath);
        std::string folderName(inFileName.substr(0, inFileName.find_last_of("/\\")));
        std::vector<std::string> fileNames;
        DebugLog << " Folder: " << folderName;
        getDataFileNames(inFileName.c_str(), folderName, fileNames);
        DebugLog << "  has " << fileNames.size() << " data";
        std::string outputFolderFullPath = inOpt.outputFolder + inOpt.resultNamePrefix;
        std::string outFileName = outputFolderFullPath + "/result.txt";
        std::string outFileFinal = outputFolderFullPath + "/resultFinal.txt";
        std::string dgFileName  = outputFolderFullPath + "/decisionGraph.obj";
        MLAlgorithms::DensityPeak dpAlgo;
        if (inOpt.reCalculateDG)
        {
            DebugLog << "******************************";
            DebugLog << " Run Density Peak Algorithm..";
            DebugLog << " 1. Load data.";
            for (int i = 0; i < fileNames.size(); i++)
            {
                ImageSpace::Image *ima = new ImageSpace::Image;
                ima->loadImageFromFile(fileNames.at(i).c_str());
                DataManager::getSingletonPtr()->addImage(ima);
            }

            // change image to vector
            int nSize = DataManager::getSingletonPtr()->getImageCount();
            int nDim = DataManager::getSingletonPtr()->getImageById()->getDimension();
            Eigen::MatrixXd data(nSize, nDim);
            double* imaData = new double[nDim];
            for (int i = 0; i < nSize; i++)
            {
                ImageSpace::Image* ima = DataManager::getSingletonPtr()->getImageById(i);
                ima->getData(imaData);
                for (int j = 0; j < nDim; j++)
                    data(i, j) = imaData[j];
                //DebugLog << data;
            }
            delete[] imaData;
            DebugLog << " 2. Data has loaded.";

            //DefaultLog << data;
            DebugLog << " 3. Begin to run density-peak algorithm..\n";
            dpAlgo.init(data);
            dpAlgo.setSCScaleParam(inOpt.dcScale);
            dpAlgo.commit();
            dpAlgo.saveResultToFile(outFileName.c_str());
            SaveOptions opt = { false, true, true, 0.0 };
            saveDecisionGraphAsObj(dgFileName.c_str(), opt, &dpAlgo);
            DebugLog << "Finish";
        }
        
        DebugLog << "***************************";
        DebugLog << " Analyse the Result data";

        dpAlgo.setGapParam(inOpt.gapScale);
        dpAlgo.readResultFromFile(outFileName.c_str());
        dpAlgo.analyseResult();


        DebugLog << "***************************";
        DebugLog << " Cluster the data";
        if (!inOpt.reCalculateDG)
        {
            // reload the image data
            DebugLog << " . Load image data.";
            for (int i = 0; i < fileNames.size(); i++)
            {
                ImageSpace::Image *ima = new ImageSpace::Image;
                ima->loadImageFromFile(fileNames.at(i).c_str());
                DataManager::getSingletonPtr()->addImage(ima);
            }

            // change image to vector
            int nSize = DataManager::getSingletonPtr()->getImageCount();
            int nDim = DataManager::getSingletonPtr()->getImageById()->getDimension();
            Eigen::MatrixXd dataInner(nSize, nDim);
            double* imaData2 = new double[nDim];
            for (int i = 0; i < nSize; i++)
            {
                ImageSpace::Image* ima = DataManager::getSingletonPtr()->getImageById(i);
                ima->getData(imaData2);
                for (int j = 0; j < nDim; j++)
                    dataInner(i, j) = imaData2[j];
                //DebugLog << data;
            }
            delete[] imaData2;
            DebugLog << " . Data has loaded.";
            dpAlgo.setData(dataInner);
        }

        DebugLog << " clustering.." << dpAlgo.getPointSize() << "*" << dpAlgo.getPointDim();
        dpAlgo.clusterAll();
        dpAlgo.saveResultToFile(outFileFinal.c_str());
        
        DebugLog << " output the clustered data..";
        std::vector<int> clusterIdx = dpAlgo.getClusterIdx();
        for (size_t i = 0; i < clusterIdx.size(); i++)
        {
            int id = clusterIdx[i];
            std::string srcFile(fileNames[i]);
            std::stringstream ss;
            ss << outputFolderFullPath << "/" << id << "_" << (srcFile.substr(srcFile.find_last_of("/\\")+1));
            FileTools::copyFile(srcFile.c_str(), ss.str().c_str());
        }
    }

    void DPTest::getDataFileNames(const char* inFileName, const std::string& folderName, std::vector<std::string>& fileNames)
    {
        std::ifstream fin(inFileName);
        if (!fin)
            return;

        int nSize=0;
        fin >> nSize;
        std::string line;
        while(std::getline(fin, line))
        {
            if (line.empty()) continue;
            std::stringstream ss;
            ss << folderName << "/" << line;
            fileNames.push_back(ss.str());
        }
    }

    void DPTest::runTests(const char* xmlFileName)
    {
        InputOptions inOpt;
        FileTools::XmlParser xml;
        xml.loadXml(xmlFileName);
        std::vector<std::string> testNames = xml.collectLevel1Nodes();
        for (int i = 0; i < testNames.size(); i++)
        {
            bool skipThis = xml.getElementValueToBoolean(testNames[i].c_str(), InputOptionNames[XML_ML_SKIP]);
            if (skipThis) continue;
            inOpt.txtFileFullPath   = xml.getElementValue(testNames[i].c_str(), InputOptionNames[XML_ML_TXTDATA_FULLPATH]);
            inOpt.outputFolder      = xml.getElementValue(testNames[i].c_str(), InputOptionNames[XML_ML_OUTPUT_FOLDER]);
            inOpt.resultNamePrefix  = xml.getElementValue(testNames[i].c_str(), InputOptionNames[XML_ML_RESULT_PREFIX]);
            inOpt.dcScale       = xml.getElementValueToDouble(testNames[i].c_str(), InputOptionNames[XML_ML_DC_SCALE]);
            inOpt.gapScale      = xml.getElementValueToDouble(testNames[i].c_str(), InputOptionNames[XML_ML_GAP_SCALE]);
            inOpt.reCalculateDG = xml.getElementValueToBoolean(testNames[i].c_str(), InputOptionNames[XML_ML_RECACULATE_DG]);

            std::string inFileName(inOpt.txtFileFullPath);
            std::string folderName(inFileName.substr(0, inFileName.find_last_of("/\\")));
            std::vector<std::string> fileNames;
            DebugLog << " Folder: " << folderName;
            getDataFileNames(inFileName.c_str(), folderName, fileNames);
            DebugLog << "  has " << fileNames.size() << " data";
            std::stringstream ssTmp;
            ssTmp << inOpt.outputFolder << "/" << inOpt.resultNamePrefix;
            std::string outputFolderFullPath(ssTmp.str());
            ssTmp.str("");
            ssTmp << inOpt.outputFolder << "/" << inOpt.resultNamePrefix << "_result.txt"; 
            std::string outFileName(ssTmp.str());
            ssTmp.str("");
            ssTmp << inOpt.outputFolder << "/" << inOpt.resultNamePrefix << "_resultFinal.txt"; 
            std::string outFileFinal(ssTmp.str());
            ssTmp.str("");
            ssTmp << inOpt.outputFolder << "/" << inOpt.resultNamePrefix << "_decisionGraph.obj"; 
            std::string dgFileName(ssTmp.str());
            MLAlgorithms::DensityPeak dpAlgo;

            DebugLog << "********************";
            DebugLog << "input txt file: ";
            DebugLog << "   " << inFileName;
            DebugLog << "output folder: ";
            DebugLog << "   " << outputFolderFullPath;
            DebugLog << "output files: ";
            DebugLog << "   " << outFileName;
            DebugLog << "   " << outFileFinal;
            DebugLog << "   " << dgFileName;

            // prepare folder
            if (FileTools::checkDir(outputFolderFullPath.c_str()) < 0)
            {
                DebugLog << "Cannot create folder.";
                return;
            }

            if (inOpt.reCalculateDG)
            {
                DebugLog << "******************************";
                DebugLog << " Run Density Peak Algorithm..";
                DebugLog << " 1. Load data.";
                for (int i = 0; i < fileNames.size(); i++)
                {
                    ImageSpace::Image *ima = new ImageSpace::Image;
                    ima->loadImageFromFile(fileNames.at(i).c_str());
                    DataManager::getSingletonPtr()->addImage(ima);
                }

                // change image to vector
                int nSize = DataManager::getSingletonPtr()->getImageCount();
                int nDim = DataManager::getSingletonPtr()->getImageById()->getDimension();
                Eigen::MatrixXd data(nSize, nDim);
                double* imaData = new double[nDim];
                for (int i = 0; i < nSize; i++)
                {
                    ImageSpace::Image* ima = DataManager::getSingletonPtr()->getImageById(i);
                    ima->getData(imaData);
                    for (int j = 0; j < nDim; j++)
                        data(i, j) = imaData[j];
                    //DebugLog << data;
                }
                delete[] imaData;
                DebugLog << " 2. Data has loaded.";

                //DefaultLog << data;
                DebugLog << " 3. Begin to run density-peak algorithm..\n";
                dpAlgo.init(data);
                dpAlgo.setSCScaleParam(inOpt.dcScale);
                dpAlgo.commit();
                dpAlgo.saveResultToFile(outFileName.c_str());
                SaveOptions opt = { false, true, true, 0.0 };
                saveDecisionGraphAsObj(dgFileName.c_str(), opt, &dpAlgo);
                DebugLog << "Finish";
            }
        
            DebugLog << "***************************";
            DebugLog << " Analyse the Result data";

            dpAlgo.setGapParam(inOpt.gapScale);
            dpAlgo.readResultFromFile(outFileName.c_str());
            dpAlgo.analyseResult();


            DebugLog << "***************************";
            DebugLog << " Cluster the data";
            if (!inOpt.reCalculateDG)
            {
                // reload the image data
                DebugLog << " . Load image data.";
                for (int i = 0; i < fileNames.size(); i++)
                {
                    ImageSpace::Image *ima = new ImageSpace::Image;
                    ima->loadImageFromFile(fileNames.at(i).c_str());
                    DataManager::getSingletonPtr()->addImage(ima);
                }

                // change image to vector
                int nSize = DataManager::getSingletonPtr()->getImageCount();
                int nDim = DataManager::getSingletonPtr()->getImageById()->getDimension();
                Eigen::MatrixXd dataInner(nSize, nDim);
                double* imaData2 = new double[nDim];
                for (int i = 0; i < nSize; i++)
                {
                    ImageSpace::Image* ima = DataManager::getSingletonPtr()->getImageById(i);
                    ima->getData(imaData2);
                    for (int j = 0; j < nDim; j++)
                        dataInner(i, j) = imaData2[j];
                    //DebugLog << data;
                }
                delete[] imaData2;
                DebugLog << " . Data has loaded.";
                dpAlgo.setData(dataInner);
            }

            DebugLog << " clustering.." << dpAlgo.getPointSize() << "*" << dpAlgo.getPointDim();
            dpAlgo.clusterAll();
            dpAlgo.saveResultToFile(outFileFinal.c_str());
        
            DebugLog << " output the clustered data..";
            std::vector<int> clusterIdx = dpAlgo.getClusterIdx();
            for (size_t i = 0; i < clusterIdx.size(); i++)
            {
                int id = clusterIdx[i];
                std::string srcFile(fileNames[i]);
                std::stringstream ss;
                ss << outputFolderFullPath << "/" << id << "_" << (srcFile.substr(srcFile.find_last_of("/\\")+1));
                FileTools::copyFile(srcFile.c_str(), ss.str().c_str());
            }
            DebugLog << "Finish.";
        }
    }
}

GLOBAL_NAMESPACE_END